# -- coding: utf-8 --
import sys
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector 
from scrapy.contrib.exporter import XmlItemExporter
from scrapy.contrib.exporter import JsonItemExporter
from scrapy.contrib.exporter import JsonLinesItemExporter
from scrapy.contrib.exporter import CsvItemExporter

from tutorial.items import AudioLibFile

class TvoxAudioLibSpider(BaseSpider):
    name = "tvoxaudiolib"
    allowed_domains = ["www.thereminvox.com"]
    start_urls = [
        "http://www.thereminvox.com/filemanager/list/2/index.html",
        "http://www.thereminvox.com/filemanager/list/5/index.html"
    ]

    def parse(self, response):
	print "RESPONSE !!!"
    	sel = HtmlXPathSelector(response)
	####### subcats
	subcats = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/div[1]/text()')
	print "sottocategorie %s \n" % (subcats[0].extract().split(' ')[-1])
	xpath_base_with_sub = 'html/body/table/tr[2]/td/table/tr[1]/td[2]/form/div[3]/div/table[2]/tr[position()>1]'
        xpath_base_without_sub = 'html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr[position()>1]'
	#if subcats[0].extract():
	if subcats[0].extract().split(' ')[-1] == "Categories:":
		xpath_base = xpath_base_with_sub 
	else:
		xpath_base = xpath_base_without_sub 

	items = []
	###### titolo
        #titles = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr/td/a/b/text()')
        titles = sel.select(xpath_base + '/td/a/b/text()')
        for title in titles:
		item = AudioLibFile()
		item['title'] = title.extract()
		items.append(item)
	###### artista
	#artisti = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr/td/a/../text()')
	artisti = sel.select(xpath_base + '/td/a/../text()')
	i = 0
	for artista in artisti:
		items[i]['artist'] = artista.extract()
		i = i + 1
	##### descrizione
	#descs = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr/td[@colspan]')
	descs = sel.select(xpath_base + '/td[@colspan]')
	i = 0
	for desc in descs:
		descline = ""
		ilines = desc.select('i/text()')
		for iline in ilines:
			descline = descline + iline.extract()
		items[i]['desc'] = descline
		i = i + 1
	##### filename	
        #filenames = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr/td/a/@href')
        filenames = sel.select(xpath_base + '/td/a/@href')
	i = 0
        for filename in filenames:
		items[i]['filename'] = filename.extract().split('/')[-1]
		i = i + 1 
	#data upload	
        #upldates = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr[position()>1]/td[2]/text()')
	upldates = sel.select(xpath_base + '/td[2]/text()')
	i = 0
        for upldate in upldates:
		items[i]['upldate'] =  upldate.extract()
		i = i + 1 
	##### dimensioni
        #sizes = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr[position()>1]/td[3]/nobr/text()')
	sizes = sel.select(xpath_base + '/td[3]/nobr/text()')
	i = 0
        for size in sizes:
		items[i]['size'] = size.extract()
		i = i + 1 
	##### proprietario
        #owners = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/form/table[2]/tr[position()>1]/td[4]/text()')
	owners = sel.select(xpath_base + '/td[4]/text()')
	i = 0
        for owner in owners:
		items[i]['owner'] = owner.extract()
		i = i + 1 
	##### labels
	labels = sel.select('html/body/table/tr[2]/td/table/tr[1]/td[2]/div[4]/a[position()>1]/text()')
	labels_list = []
	for label in labels:
		labels_list.append(label.extract())
	labels_str = ", ".join(labels_list)
	for item in items:
		item['labels'] = labels_str
	####### stampa items
	#i = 0
	#for item in items:
	#	i = i + 1
	#	print "Item n°  %d: " %  i
	#	#print "titolo: %s " %  item['title']
	#	#print "descrizione: %s " %  item['desc']
	#	#print "artista: %s" %  item['artist']
	#	print "filename: %s" %  item['filename']
	#	#print "data di upload: %s" %  item['upldate']
	#	#print "dimensioni: %u" %  item['size']
	#	#print "proprieario: %s" %  item['owner']
	#	print "labels: %s" % item['labels']
	#	#print "\n"

       	file = open('tvoxaudiolibfiles.json', 'a')
       	exporter = JsonLinesItemExporter(file)
       	exporter.start_exporting()

	for item in items:
       		exporter.export_item(item)

       	exporter.finish_exporting()
